{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = 'music/Data/'\n",
    "# 用户和item索引\n",
    "users_index = cPickle.load(open(dpath+'users_index.pkl','rb'))\n",
    "items_index = cPickle.load(open(dpath+'items_index.pkl','rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "\n",
    "# 到排序\n",
    "#每个用户打过分的电影\n",
    "user_items = cPickle.load(open(dpath+'user_items.pkl','rb'))\n",
    "item_users = cPickle.load(open(dpath+'item_users.pkl','rb'))\n",
    "\n",
    "# 用户关系矩阵\n",
    "user_item_scores = sio.mmread(dpath+'user_item_scores')\n",
    "user_item_scores = user_item_scores.tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每个用户的平均打分\n",
    "user_ms = np.zeros(n_users)\n",
    "\n",
    "for i in range(n_users):\n",
    "    score = 0;\n",
    "    n_item = 0;\n",
    "    for item in user_items[i]:\n",
    "        score += user_item_scores[i,item]\n",
    "        n_item += 1\n",
    "    if n_item > 0:\n",
    "        user_ms[i] = score/n_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def item_similarity(iid1,iid2):\n",
    "    su = {}\n",
    "    for user in item_users[iid1]: # 找出item1打分的用户\n",
    "        if user in item_users[iid2]:  # 如果item1打分的用户也再item2中则为有效的有效的用户\n",
    "            su[user] = 1\n",
    "    n = len(su)\n",
    "    if n == 0:\n",
    "        similarity = 0.0\n",
    "        return similarity\n",
    "    # 对item1的有效打分\n",
    "    s1 = np.array([user_item_scores[user,iid1]-user_ms[user] for user in su])\n",
    "    # 对item2的有效打分\n",
    "    s2 = np.array([user_item_scores[user,iid2]-user_ms[user] for user in su])\n",
    "    \n",
    "    similarity = 1 - ssd.cosine(s1,s2)\n",
    "    \n",
    "    if np.isnan(similarity):\n",
    "        similarity = 0.0\n",
    "    return similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ii = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jay/.local/lib/python3.7/site-packages/scipy/spatial/distance.py:720: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ii = 100\n",
      "ii = 200\n",
      "ii = 300\n",
      "ii = 400\n",
      "ii = 500\n",
      "ii = 600\n",
      "ii = 700\n"
     ]
    }
   ],
   "source": [
    "# 预算好所有物品之间的相似度，可以加快后边的计算\n",
    "# 对于物品较少，物品比较固定的系统实用\n",
    "item_similarity_matrix = np.matrix(np.zeros(shape = (n_items,n_items)),float)\n",
    "\n",
    "for ii in range(n_items):\n",
    "    item_similarity_matrix[ii,ii] = 1.0\n",
    "    if ii % 100 == 0:\n",
    "        print('ii = {}'.format(ii))\n",
    "    for ij in range(ii+1,n_items):\n",
    "        item_similarity_matrix[ij,ii]= item_similarity(ii,ij)\n",
    "        item_similarity_matrix[ii,ij] = item_similarity_matrix[ij,ii]\n",
    "cPickle.dump(item_similarity_matrix,open(dpath+'item_similarity_matrix.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测用户对item的打分\n",
    "# 利用用户打过分的item计算相似度\n",
    "def Item_CF_pred(uid,iid):\n",
    "    sim_accumnlate = 0.0\n",
    "    rat_acc = 0.0\n",
    "    for item_id in user_items[uid]: # 这个用户对哪些物品打过分\n",
    "        # 获取这个物品之间的相识度\n",
    "        sim = item_similarity_matrix[item_id,iid]\n",
    "        if sim != 0:\n",
    "            rat_acc += sim * user_item_scores[uid,item_id]\n",
    "            sim_accumnlate += np.abs(sim)\n",
    "    # 估计最后结果，如果为0就给均分\n",
    "    if sim_accumnlate != 0:\n",
    "        score = rat_acc/sim_accumnlate\n",
    "    else:\n",
    "        score = user_ms[uid]\n",
    "    \n",
    "    if score < 0:\n",
    "        score = 0.0\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对物品进行推荐\n",
    "\n",
    "def recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    # 训练集中用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "    \n",
    "    # 该用户对所有物品打分\n",
    "    user_item_scores = np.zeros(n_items)\n",
    "    \n",
    "    # 预测打分\n",
    "    for i in range(n_items):\n",
    "        if i not in cur_user_items:\n",
    "            user_item_scores[i] = Item_CF_pred(cur_user_id,i)\n",
    "    # 用元组来存（分数，物品id）\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_item_scores))),reverse=True)\n",
    "    columns = ['song','score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "    \n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        # 把index转化乘list然后通过index定位value所在位置，然后再将key（物品真正的id）转化成list，找到真正的item id\n",
    "        cur_item = list(items_index.keys())[list(items_index.values()).index(cur_item_index)]\n",
    "        \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)] = [cur_item,sort_index[i][0]]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>30628</td>\n",
       "      <td>b7c24f770be6b802805ac0e2106624a517643c17</td>\n",
       "      <td>SOEBOWM12AB017F279</td>\n",
       "      <td>10.810811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>31485</td>\n",
       "      <td>9254a3fdc569428c3b1c3904db36d485c47e2544</td>\n",
       "      <td>SOPXKYD12A6D4FA876</td>\n",
       "      <td>23.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>16920</td>\n",
       "      <td>31aad1036a404737ee8b88ea2da68813c9a46874</td>\n",
       "      <td>SOQJHUW12AB0188A24</td>\n",
       "      <td>2.678571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>16954</td>\n",
       "      <td>e3e8103d0751e29693f9b03a58efa5c21acf2115</td>\n",
       "      <td>SODJWHY12A8C142CCE</td>\n",
       "      <td>17.241379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>25098</td>\n",
       "      <td>520bb6f7bd2fc51d02f236398acdc5170cc299a8</td>\n",
       "      <td>SOWNVIV12AB0184846</td>\n",
       "      <td>12.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                      user                song  \\\n",
       "0       30628  b7c24f770be6b802805ac0e2106624a517643c17  SOEBOWM12AB017F279   \n",
       "1       31485  9254a3fdc569428c3b1c3904db36d485c47e2544  SOPXKYD12A6D4FA876   \n",
       "2       16920  31aad1036a404737ee8b88ea2da68813c9a46874  SOQJHUW12AB0188A24   \n",
       "3       16954  e3e8103d0751e29693f9b03a58efa5c21acf2115  SODJWHY12A8C142CCE   \n",
       "4       25098  520bb6f7bd2fc51d02f236398acdc5170cc299a8  SOWNVIV12AB0184846   \n",
       "\n",
       "       score  \n",
       "0  10.810811  \n",
       "1  23.076923  \n",
       "2   2.678571  \n",
       "3  17.241379  \n",
       "4  12.500000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "df_triplet_test = pd.read_csv(dpath+'test.csv')\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6b3d5eaba2e55699cb725d0c605c6eca1b302dfe is new user\n",
      "a3a9329463c55f63876f84b0c47b4f90ca9db7bc is new user\n",
      "de27b74444dae039f76e421362c6a914da9f8b41 is new user\n",
      "6a58f480d522814c087fd3f8c77b3f32bb161f9d is new user\n",
      "52a6c7b6221f57c89dacbbd06854ca0dc415e9e6 is new user\n",
      "467e0e46181933c7e1a936e513ca55fbab4edaed is new user\n"
     ]
    }
   ],
   "source": [
    "# 统计总的用户\n",
    "unique_users_test = df_triplet_test['user'].unique()\n",
    "\n",
    "# 为每个用户推荐10个商品\n",
    "n_rec_items = 10\n",
    "\n",
    "#性能评价计算精确率和召回率\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合，用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "# 残差平方和，用于计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "# 对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    if user not in users_index:\n",
    "        print('{} is new user'.format(user))\n",
    "        continue\n",
    "    user_records_test = df_triplet_test[df_triplet_test.user == user]\n",
    "    \n",
    "    rec_items = recommend(user)\n",
    "    \n",
    "    for i in range(n_rec_items):\n",
    "        item = rec_items.iloc[i]['song']\n",
    "        \n",
    "        if item in user_records_test['song'].values:\n",
    "            n_hits +=1\n",
    "            \n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    # 计算rmse\n",
    "    for i in range(user_records_test.shape[0]):\n",
    "        item = user_records_test.iloc[i]['song']\n",
    "        score = user_records_test.iloc[i]['score']\n",
    "        \n",
    "        df1 = rec_items[rec_items.song == item]\n",
    "        if df1.shape[0] == 0:\n",
    "            print('{} is new item. '.format(item))\n",
    "            continue\n",
    "        pre_score = df1['score'].values[0]\n",
    "        rss_test += (pre_score - score)**2\n",
    "    # 推荐item 总数\n",
    "    n_total_rec_items += n_rec_items\n",
    "    n_test_items += user_records_test.shape[0]\n",
    "\n",
    "precision = n_hits / (1.0 * n_total_rec_items)\n",
    "recall = n_hits / (1.0 * n_test_items)\n",
    "\n",
    "# 覆盖率\n",
    "coverage = len(all_rec_items) / (1.0 * n_items)\n",
    "\n",
    "# 打分均方误差\n",
    "rmse = np.sqrt(rss_test/df_triplet_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.012517193947730399"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.012136569751933848"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.055950681020107"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
